#!/usr/bin/env python3
##############################################################################
# File: ScrapeMarketData.py
#
# Author:  Ryan Luu     ryanluu@gmail.com
#
# Description:
#   Scrapes market data from TFC Commodity Charts (http://tfc-charts.w2d.com/)
#
# Usage:
#   ./ScrapeMarketData.py --help
#   ./ScrapeMarketData.py --directory=/home/ryan/marketData --all
#
##############################################################################

# For obtaining current directory path information, and creating directories
import os, sys 
import errno

# For logging.
import logging
import logging.handlers

# For parsing command-line options
from optparse import OptionParser  

# For connecting to the server via HTTP
import http.cookiejar
import urllib
from urllib import *

# Regular expressions.
import re

# For sleeping between HTTP GETs.
import time

# For getting a random number for random sleep times.
import random

# For comparing dates.
import datetime

from subprocess import call

##############################################################################
# Global variables

# Version string.
VERSION = "0.9"

# Logger object.  Will be set when the logger object is created.
log = None

# Options object.  Will hold the data from the command-line arguments.
# The object will be set when created.
options = None

# Cookie preferences are required for the pricebar data to be returned.
defaultHttpHeaders = \
        [('User-agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.0.10) Gecko/2009042315 Firefox/3.0.10'),
         ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
         ('Accept-Language', 'en-us,en;q=0.5'),
         ('Accept-Charset', 'ISO-8859-1;q=0.7,*;q=0.7'),
         ('Cookie', 'tcuid=782990; CCIPCntry=-; Czdpref=n%2Bf%2Bf%2Bt%2Bt%2Bt%2B')]
# Notes on cookies:
#   - Czdpref holds information about what info to return.
#   - CCIPCntry: Not sure.
#   - tcuid is also required.


# Minimum sleep time (in seconds) between HTTP calls.  Default values
# are set via the command-line options parser and the values can be
# overwritten from the command-line.
minimumSleepTimeSec = -1

# Sleep time variance (in seconds) between HTTP calls.  Default values
# are set via the command-line options parser and the values can be
# overwritten from the command-line.
sleepTimeVarianceSec = -1

##############################################################################

# Class that holds info about contracts' name and symbol.
class ContractBasicInfo:
    def __init__(self, contractName, contractSymbol):
        self.name = contractName
        self.symbol = contractSymbol


# Class that holds information about the contract specifications.
class ContractSpecs:
    def __init__(self,
                 contractName, symbol, months, tickSize,
                 quotedUnits, tradingUnit, minFluc, initMargin, maintMargin):
        self.contractName = contractName
        self.symbol = symbol
        self.months = months
        self.tickSize = tickSize
        self.quotedUnits = quotedUnits
        self.tradingUnit = tradingUnit
        self.minFluc = minFluc
        self.initMargin = initMargin
        self.maintMargin = maintMargin

# Class that holds detailed data about a certain futures contract.
class ContractInfo:
    def __init__(self, 
                 contractNameStr, 
                 contractSymbolStr, 
                 contractDescStr, 
                 urlPathStr):
        global log
        self.contractName = contractNameStr
        self.contractSymbol = contractSymbolStr
        self.contractMonth = ""
        self.contractYear = ""
        self.urlPath = urlPathStr

        contractDescStr = contractDescStr.strip()
        contractDescStr = contractDescStr.replace(":", "")
        contractDescStr = contractDescStr.replace("* ", "")

        if re.search("Weekly", contractDescStr):
            # Weekly chart.
            log.debug("Found weekly.")
            self.contractMonth = "Weekly"
        elif re.search("Monthly", contractDescStr):
            # Monthly chart.
            log.debug("Found monthly.")
            self.contractMonth = "Monthly"
        elif re.search("Daily", contractDescStr):
            # Daily chart, for commodity and stock indexes.
            log.debug("Found daily.")
            self.contractMonth = "Daily"
        elif contractDescStr.lower().startswith("cash"):
            self.contractMonth = "Y"
        else:
            splitted = contractDescStr.split(" ")
            monthStr = ""
            if len(splitted) == 2:
                monthStr = splitted[0].lower()
                self.contractYear = splitted[1]
                log.debug("monthStr == " + monthStr)
            else:
                log.error("Failed while trying to do parsing on contract " + 
                          "description string: " + contractDescStr)
                sys.exit(1)

            if monthStr.startswith("jan"):
                self.contractMonth="F"
            elif monthStr.startswith("feb"):
                self.contractMonth="G"
            elif monthStr.startswith("mar"):
                self.contractMonth="H"
            elif monthStr.startswith("apr"):
                self.contractMonth="J"
            elif monthStr.startswith("may"):
                self.contractMonth="K"
            elif monthStr.startswith("jun"):
                self.contractMonth="M"
            elif monthStr.startswith("jul"):
                self.contractMonth="N"
            elif monthStr.startswith("aug"):
                self.contractMonth="Q"
            elif monthStr.startswith("sep"):
                self.contractMonth="U"
            elif monthStr.startswith("oct"):
                self.contractMonth="V"
            elif monthStr.startswith("nov"):
                self.contractMonth="X"
            elif monthStr.startswith("dec"):
                self.contractMonth="Z"
            else:
                log.error("Failed to determine what month this contract is.")
                sys.exit(1)

        #log.debug("self.contractMonth == " + self.contractMonth)
        #log.debug("self.contractYear == " + self.contractYear)
        #log.debug(self.monthStrAs2DigitInteger())
        #log.debug(self.monthStrAs1CharStd())
        #log.debug(self.monthStrAs1CharPriceDataStd())
        #log.debug(self.yearAs1DigitString())
        #log.debug(self.yearAs2DigitString())
        #log.debug(self.yearAs4DigitString())
        #log.debug(self.getUrl())
        #log.debug(self.toString())
        #log.debug("-----------")


    # Returns a string of the month as a 2-character integer string.
    # e.g. January would be "01" and December would be "12".
    def monthStrAs2DigitInteger(self):
        global log
        if self.contractMonth == "F":
            return "01"
        elif self.contractMonth == "G":
            return "02"
        elif self.contractMonth == "H":
            return "03"
        elif self.contractMonth == "J":
            return "04"
        elif self.contractMonth == "K":
            return "05"
        elif self.contractMonth == "M":
            return "06"
        elif self.contractMonth == "N":
            return "07"
        elif self.contractMonth == "Q":
            return "08"
        elif self.contractMonth == "U":
            return "09"
        elif self.contractMonth == "V":
            return "10"
        elif self.contractMonth == "X":
            return "11"
        elif self.contractMonth == "Z":
            return "12"
        else:
            log.error("Invalid month symbol for conversion to 2-digit " +
                      "integer string: " + self.contractMonth)
            return "??"


    # Returns a string of the month in the standard format.
    # e.g., "Z" for December, "H" for March, etc.
    def monthStrAs1CharStd(self):
        global log
        if len(self.contractMonth) == 1:
            return self.contractMonth
        else:
            log.error("Invalid contract month string for conversion to " + 
                      "1-char standard: " + self.contractMonth)
            return "?"
    
    def monthAs3LetterStr(self):
        global log
        if self.contractMonth == "F":
            return "Jan"
        elif self.contractMonth == "G":
            return "Feb"
        elif self.contractMonth == "H":
            return "Mar"
        elif self.contractMonth == "J":
            return "Apr"
        elif self.contractMonth == "K":
            return "May"
        elif self.contractMonth == "M":
            return "Jun"
        elif self.contractMonth == "N":
            return "Jul"
        elif self.contractMonth == "Q":
            return "Aug"
        elif self.contractMonth == "U":
            return "Sep"
        elif self.contractMonth == "V":
            return "Oct"
        elif self.contractMonth == "X":
            return "Nov"
        elif self.contractMonth == "Z":
            return "Dec"
        else:
            log.error("Invalid month symbol for conversion to 3-letter " +
                      "string: " + self.contractMonth)
            return "???"

    # Returns a string of the month in the format used by Price-Data,
    # which is "1" for January, "A" for october
    def monthStrAs1CharPriceDataStd(self):
        global log
        if self.contractMonth == "F":
            return "1"
        elif self.contractMonth == "G":
            return "2"
        elif self.contractMonth == "H":
            return "3"
        elif self.contractMonth == "J":
            return "4"
        elif self.contractMonth == "K":
            return "5"
        elif self.contractMonth == "M":
            return "6"
        elif self.contractMonth == "N":
            return "7"
        elif self.contractMonth == "Q":
            return "8"
        elif self.contractMonth == "U":
            return "9"
        elif self.contractMonth == "V":
            return "A"
        elif self.contractMonth == "X":
            return "B"
        elif self.contractMonth == "Z":
            return "C"
        else:
            log.error("Invalid month symbol for conversion to 2-digit " +
                      "integer string: " + self.contractMonth)
            return "?"

    # Return a string with the year as 1 digit.  This actually comes
    # out to be (year % 10), printed as a string.  E.g., year 2009
    # gets returned as "9".
    def yearAs1DigitString(self):
        if len(self.contractYear) >= 1:
            return self.contractYear[-1:]
        else:
            log.error("Not enough digits for 1-digit year in: " +
                      self.contractYear)
            return "?"
    
    # Return a string with the year as 2 digits.  This actually comes
    # out to be (year % 100), printed as a string.  E.g., year 2009
    # gets returned as "09".
    def yearAs2DigitString(self):
        if len(self.contractYear) >= 2:
            return self.contractYear[-2:]
        else:
            log.error("Not enough digits for 2-digit year in: " +
                      self.contractYear)
            return "?"

    # Return a string with the year as 4 digits.  This is just the
    # string format of the year.
    def yearAs4DigitString(self):
        if len(self.contractYear) >= 4:
            return self.contractYear[-4:]
        else:
            log.error("Not enough digits for 4-digit year in: " +
                      self.contractYear)
            return "?"

    # Return a string of URL to get this chart data.
    def getUrl(self):
        return "http://tfc-charts.w2d.com" + self.urlPath

    # Return the string representation of this object.
    def toString(self):
        # Because we have data for weekly and monthly time periods,
        # and they do not fall into the category of being a futures
        # contract, we need special cases for dealing with different
        # fields.
        monthStr = ""
        yearStr = ""
        monthAndYearStr = ""
        
        if self.contractMonth == "Weekly" or \
               self.contractMonth == "Monthly" or \
               self.contractMonth == "Daily":
            monthStr = "Period: " + self.contractMonth
            yearStr = ""
        else:
            monthStr = "Month: " + self.contractMonth + \
                       " (" + self.monthAs3LetterStr() + ")"
            yearStr = "Year: " + self.contractYear

        if yearStr == "":
            monthAndYearStr = monthStr
        else:
            monthAndYearStr = monthStr  + ", " + yearStr

        # Assemble the returned string.
        rv = "Name: " + self.contractName + \
             ", Symbol: " + self.contractSymbol + \
             ", " + monthAndYearStr + \
             ", URL: " + self.getUrl()
        return rv

    # The Python way of implicitly turning this into a string.
    def __str__(self):
        return self.toString()
    
# Class that holds information about the bar of price data.
# Inputs should all be strings in the format:
# year:  XXXX
# month: XX
# day: XX
class PriceBar:
    # This corresponds to the fields given in columnNames.
    NumFieldsExpected = 7
    
    def __init__(self,
                 year, month, day,
                 openPrice, highPrice, lowPrice, closePrice,
                 volume=0, openInt=0):
        self.year = year
        self.month = month
        self.day = day
        self.openPrice = openPrice
        self.highPrice = highPrice
        self.lowPrice = lowPrice
        self.closePrice = closePrice
        self.volume = volume
        self.openInt = openInt


    def __eq__(self, other):
        if (self.year == other.year and
            self.month == other.month and
            self.day == other.day and
            self.openPrice == other.openPrice and
            self.highPrice == other.highPrice and
            self.lowPrice == other.lowPrice and
            self.closePrice == other.closePrice and
            self.volume == other.volume and
            self.openInt == other.openInt):
            return True
        else:
            return False

    def __ne__(self, other):
        return not (__eq__(self, other))
    
    # Returns the column names that toString() will return the string in.
    def columnNames(self):
        columnNames = "\"Date\",\"Open\",\"High\",\"Low\",\"Close\",\"Volume\",\"OpenInt\""
        return columnNames

    # Returns the date in mm/dd/yyyy string format.
    def getUSDateStr(self):
        return self.month + "/" + self.day + "/" + self.year

    # Returns the date as a datetime.date object.
    def date(self):
        rv = datetime.date(int(self.year), int(self.month), int(self.day))
        return rv
    
    # Returns the price-bar data in a CSV string format.
    def toPriceBarCSVStr(self):
        strFormat = "{m:2}/{d:2}/{y:4},{o},{h},{l},{c},{v},{oi}"
        return strFormat.format(m=self.month, d=self.day, y=self.year,
                                o=self.openPrice, h=self.highPrice,
                                l=self.lowPrice, c=self.closePrice,
                                v=self.volume, oi=self.openInt)

    def toString(self):
        return self.toPriceBarCSVStr()
        
    # The Python way of implicitly turning this into a string.
    def __str__(self):
        return self.toString()
    
##############################################################################
# Subroutine: parseCommandlineArgs()
#
# Description:
#    Parses the arguments specified on the command-line.
#
# Returns:
#    (options, args) for the options and arguments passed.
#
def parseCommandlineArgs():
    global VERSION
    
    # Create the parser
    parser = OptionParser()

    # Specify all valid options.
    parser.add_option("-v", "--version",
                      action="store_true",
                      dest="version",
                      default=False,
                      help="Display script version info and author contact.")
    
    parser.add_option("--log-level",
                      action="store",
                      type="string",
                      dest="logLevel",
                      default="info",
                      help="Specify a log level to use.  Valid choices for " +
                      "the log level are debug, info, warning, error, " +
                      "and critical.  Default is info.",
                      metavar="<LEVEL>")

    parser.add_option("--min-sleep",
                      action="store",
                      type="float",
                      dest="minSleepSec",
                      default=12.0,
                      help="Specify a minimum number of seconds of " + \
                           "sleep time between HTTP calls.  Default is 12.  " +
                           "DO NOT USE THIS SETTING UNLESS YOU KNOW WHAT YOU " +
                           "ARE DOING.",
                      metavar="<SECONDS>")
    
    parser.add_option("--sleep-variance",
                      action="store",
                      type="float",
                      dest="sleepVarianceSec",
                      default=5.0,
                      help="Specify a number of seconds of sleep time " +
                           "variance between HTTP calls.  Default is 5.  " +
                           "DO NOT USE THIS SETTING UNLESS YOU KNOW WHAT YOU " +
                           "ARE DOING.",
                      metavar="<SECONDS>")
    
    parser.add_option("-d", "--directory",
                      action="store",
                      type="string",
                      dest="dataDirectory",
                      default=os.path.join(sys.path[0], "data"),
                      help="Location to create directories and update data " +
                      "files.  If a directory is not specified, then by " +
                      "default, a 'data' subdirectory will be created " +
                      "and used where this script resides.",
                      metavar="<DIRECTORY>")

    parser.add_option("-a", "--all",
                      action="store_true",
                      dest="all",
                      default=False,
                      help="Retrieve data for all available contracts.")

    parser.add_option("--agricultural",
                      action="store_true",
                      dest="agricultural",
                      default=False,
                      help="Retrieve data for agricultural contracts.")

    parser.add_option("--currencies",
                      action="store_true",
                      dest="currencies",
                      default=False,
                      help="Retrieve data for currency contracts.")

    parser.add_option("--livestock",
                      action="store_true",
                      dest="livestock",
                      default=False,
                      help="Retrieve data for livestock contracts.")

    parser.add_option("--food",
                      action="store_true",
                      dest="food",
                      default=False,
                      help="Retrieve data for food/fiber/softs contracts.")

    parser.add_option("--financial",
                      action="store_true",
                      dest="financial",
                      default=False,
                      help="Retrieve data for financial contracts.")

    parser.add_option("--index",
                      action="store_true",
                      dest="index",
                      default=False,
                      help="Retrieve data for index contracts.")

    parser.add_option("--energy",
                      action="store_true",
                      dest="energy",
                      default=False,
                      help="Retrieve data for oil/energy contracts.")

    parser.add_option("--metals",
                      action="store_true",
                      dest="metals",
                      default=False,
                      help="Retrieve data for metals contracts.")

    parser.add_option("--indexes",
                      action="store_true",
                      dest="indexes",
                      default=False,
                      help="Retrieve data for commodity and stock indexes.")
    
    # Parse the arguments into options.
    global options
    (options, args) = parser.parse_args()

    # Print version information if the flag was used.
    if (options.version == True):
        print("ScrapeMarketData.py (Version " + VERSION + ")")
        print("By Ryan Luu, ryanluu@gmail.com")
        sys.exit(0)
    
    # Verify that at least one futures type is specified for which to
    # gather futures market data.
    if (options.all == False and
        options.agricultural == False and
        options.currencies == False and
        options.livestock == False and
        options.food == False and
        options.financial == False and
        options.index == False and
        options.energy == False and
        options.metals == False and
        options.indexes == False):
        # None of the options were specified.  Print an error message
        # and show the usage info.
        print("ERROR:  You must select the contract types for which you " +
              "would like data to be collected.  Re-run with --help to " +
              "get a list of valid options.")
        sys.exit(1)

    # Create the data directory if it does not exist.
    try:
        os.makedirs(options.dataDirectory)
    except os.error as e:
        if e.errno != errno.EEXIST:
            raise

    #print("DEBUG:  Current path of script is ", sys.path[0])
    #print("DEBUG:  Base directory of data is set to ", options.dataDirectory)

    return (options, args)


##############################################################################
# Subroutine: getSupportedContractSymbolsHTML()
#
# Description:
#   Does a HTTP get to obtain html from a webpage that has info
#   regarding the supported contract symbols we can obtain data for.
#
# Returns:
#   A string containing the HTML.
#
def getSupportedContractSymbolsHTML():
    webpage = "http://tfc-charts.w2d.com/menu.html"
    log.info("Getting list of supported futures symbols from " + webpage)

    log.debug("getSupportedContractSymbolsHTML(): Setting up HTTP connection " +
              "to " + webpage)

    opener = urllib.request.build_opener()
    request = urllib.request.Request(webpage)
    opener.addheaders = defaultHttpHeaders

    log.debug("getSupportedContractSymbolsHTML(): Opening HTTP request.")
    response = opener.open(request)

    log.debug("getSupportedContractSymbolsHTML(): Reading HTTP response.")
    data = response.read().decode()

    log.debug("Data read from {} is: ***{}***".format(webpage, data))
    
    # Remove converted newline characters.
    #data = data.replace('\\r\\n', '')
    #data = data.replace('\\n', '')

    return data
    

##############################################################################
# Subroutine: getSupportedContractsHTML()
#
# Description:
#   Does a HTTP get to obtain html from a webpage that has info
#   regarding the supported contracts we can obtain data for.
#
# Returns:
#   A string containing the HTML.
#
def getSupportedContractsHTML():
    webpage = "http://tfc-charts.w2d.com/specs/summary.html"
    log.info("Getting list of supported futures contracts from " + webpage)

    log.debug("getSupportedContractsHTML(): Setting up HTTP connection " +
              "to " + webpage)

    opener = urllib.request.build_opener()
    request = urllib.request.Request(webpage)
    opener.addheaders = defaultHttpHeaders

    log.debug("getSupportedContractsHTML(): Opening HTTP request.")
    response = opener.open(request)

    log.debug("getSupportedContractsHTML(): Reading HTTP response.")
    data = str(response.read())

    # Remove converted newline characters.
    data = data.replace('\\r\\n', '')
    data = data.replace('\\n', '')
    
    return data

    

##############################################################################
# Subroutine: extractTableDataFromRow(text)
#
# Description:
#    Extracts the table data (<td>) text from the given text, and
#    returns the data as a list of strings, each string being the text
#    between the <td></td> tags.
#
def extractTableDataFromRow(text):
    data = []
    
    # Split by table data.
    tableDatas = text.split("<td")

    if len(tableDatas) > 0:
        if (tableDatas[0].find("</td>") == -1):
            tableDatas = tableDatas[1:]

    for field in tableDatas:
        field = field[field.find(">")+1:]
        field = str.strip(field)
        #log.debug("extractTableDataFromRow(): field is " + field)
        
        while (field.find("<") != -1):
            startLoc = field.find("<")
            endLoc = field.find(">")
            field = field[:startLoc] + field[endLoc+1:]
            field = str.strip(field)
            #log.debug("extractTableDataFromRow(): new field is " + field)

        #log.debug("extractTableDataFromRow(): finished with field, and " +
        #          "end result is: " + field)
        data.append(field)

    return data



##############################################################################
#
# Subroutine:  getContractBasicInfosForType(dataStr, contractType)
#
# Description:
#   Returns a list of ContractBasicInfo from the given inputs.
#
# Inputs:
#   dataStr      - String of html as returned from the web server
#
#   contractType - String containing the name of the heading
#                  that appears right before the desired table.
#
def getContractBasicInfosForType(dataStr, contractType):
    log.debug("getContractBasicInfosForType(): Processing contract type: " +
              contractType)
    
    # List of ContractBasicInfo objeccts that get returned.
    contractBasicInfos = []
    
    # Convert encoded data to a string.
    data = str(dataStr)

    # Find the location of the contract type.
    contractTypeLoc = data.find(contractType)

    # Verify the contract type string was found.
    if (contractTypeLoc == -1):
        log.error("getContractBasicInfosForType(): " +
                  "Couldn't find contract type string: " + contractType)
        # Return an empty contractBasicInfos list.
        return contractBasicInfos

    log.debug("contractTypeLoc == " + str(contractTypeLoc))
    
    # Find the next location of our end-identifier.  endTableStr is
    # actually a minomer because it turns out, the next </table> tag
    # is near the bottom of the file.  Red bold fonts separate the
    # headings, so I will just search for that.
    endTableStr = "<font color=\"red"
    endTableStr = "  </table>\n  </td>\n  </tr>\n  </table>"
    endTableStr2 = "      </table>\n    </td>\n  </tr>\n</table>"

    log.debug("Data w/o newlines is: ***{}***".format(data))
    
    log.debug("Looking for endTableStr, which is '{}'".format(endTableStr))
    
    endTableStrLoc = data.find(endTableStr, contractTypeLoc)
    endTableStr2Loc = data.find(endTableStr2, contractTypeLoc)

    log.debug("endTableStrLoc == " + str(endTableStrLoc))
    log.debug("endTableStr2Loc == " + str(endTableStr2Loc))

    # If can't find the first end table string, then fallback onto the second.
    if endTableStrLoc == -1:
        endTableStrLoc = endTableStr2Loc
        
    if endTableStrLoc == -1:
        log.error("getContractBasicInfosForType(): " +
                  "Couldn't find end </table> tag after contractType string.")
        
        # Return an empty contractBasicInfos list.
        return contractBasicInfos
    
    tableData = data[contractTypeLoc:endTableStrLoc]

    log.debug("tableData string is: ***{}***".format(tableData))

    log.debug("Now looking for <tr> string, and splitting across them.")
    
    startTableRowStr = "<tr"
    endTableRowStr = "</tr>"

    tableRows = tableData.split(startTableRowStr)

    log.debug("tableData split into " + str(len(tableRows)) + " tableRows.")

    # Verify that there is at least 1 real row containing a contract.
    # I check for <= 2 because the first table row is before the first
    # <tr>, and the next one after that is for the headings.
    if len(tableRows) <= 2:
        log.error("getContractBasicInfosForType(): " +
                  "There appears to be no table rows in this table.")
        # Return an empty contractBasicInfos list.
        return contractBasicInfos


    log.debug("Chopping off first two rows, since they never contain relevant data")
    
    tableRows = tableRows[2:]

    for tableRow in tableRows:
        log.debug("Table row is: ***{}***".format(tableRow))
    
    for tableRow in tableRows:
        log.debug("Okay.  Now working on row: ***{}***".format(tableRow))

        tableDatas = tableRow.split("</td>")

        contractName = ""
        contractSymbol = ""
        
        columnIndex = 0
        
        for tableData in tableDatas:
            log.debug("Table data is: " + tableData)
            match = re.search(r"""<td[^>]*?>(.*?)$""", tableData)
            if (match):
                cellText = match.groups()[0]

                log.debug("Cell text is: " + cellText)

                cellText = cellText.replace("<b>", "")
                cellText = cellText.replace("</b>", "")

                if columnIndex == 0:
                    log.debug("Setting contract name to cell text: " + cellText)
                    contractName = cellText
                else:
                    # Try to get the symbol, which is embedded in the
                    # URL at the very end.
                    match = re.search(r"""<a href=".*?/chart/([^"]*?)">Chart</a>""", cellText)

                    # The below comment is so that emacs Python color parsing
                    # stops thinking the double-quote is still 'on'. 
                    #
                                                                      
                    if (match):
                        contractSymbol = match.groups()[0]

                        if contractSymbol.endswith("/"):
                            # Chop off the trailing "/".
                            contractSymbol = contractSymbol[:-1]
                            
                        log.debug("Matched.  Setting contractSymbol " +
                                  "to matched str: " + contractSymbol)
                                  
                        if contractName != "":
                            # Create a ContractBasicInfo
                            cbi = ContractBasicInfo(contractName,
                                                    contractSymbol)
                            contractBasicInfos.append(cbi)
                        else:
                            log.warning("Something is wrong because we got " +
                                        "a contract symbol but no contract " +
                                        "name yet.  Continuing...")
                    else:
                        log.debug("No match.")
            else:
                # There are two valid situation for this case, and that
                # is if it is the last item in the split, which ends
                # up being the one with the closing </tr>.  If it
                # doesn't fit this case, then print out a warning.
                exceptionStr1 = "</tr>"
                exceptionStr2 = "<th colspan=\"13\" align=\"center\" bgcolor="
                exceptionStr3 = """<td valign="top">
     <table width="423" border="0" cellspacing="0" cellpadding="2" class="fe_quote_links">"""
                exceptionStr4 = """<td valign="top">
      <table width="423" border="0" cellspacing="0" cellpadding="2" class="fe_quote_links">"""
                if (tableData.find(exceptionStr1) == -1 and
                    tableData.find(exceptionStr2) == -1 and
                    tableData.find(exceptionStr3) == -1 and
                    tableData.find(exceptionStr4) == -1):
                    log.warning("Could not match regex string for table " +
                                "data: " + tableData + ".  Continuing.")

            # Increment counter for column index.
            columnIndex += 1

    log.debug("Returning " + str(len(contractBasicInfos)) +
              " number of ContractBasicInfos")
    return contractBasicInfos

##############################################################################
#
# Subroutine: getContractSpecsForContractType(dataStr, contractType)
#
# Description:
#   Returns a list of ContractSpecs from the given inputs.
#
# Inputs:
#   dataStr      - String of html as returned from the web server
#
#   contractType - String containing the name of the heading
#                  that appears right before the desired table.
#
def getContractSpecsForContractType(dataStr, contractType):
    # List of ContractSpecs objects for the supported contracts.
    contractSpecs = []

    # Convert encoded data to a string.
    data = str(dataStr)

    # Find the location of the contract type.
    contractTypeLoc = data.find(contractType)

    # Verify the contract type string was found.
    if (contractTypeLoc == -1):
        log.error("Couldn't find contract type string: " + contractType)
        sys.exit(1)

    #log.debug("getContractSpecsForContractType(): contractTypeLoc == " +
    #          str(contractTypeLoc))

    # Find the next locations of <table> and </table> respectively, and
    # make sure the location for <table> is less than the location for
    # </table>.  
    startTableStr = "<table"
    endTableStr = "</table>"

    # Make a lower-case version of the data, since <table> could be
    # capitalized.
    lowercaseData = data.lower()

    # Find startTableStr, starting from location 'contractTypeLoc'
    startTableLoc = lowercaseData.find(startTableStr, contractTypeLoc)

    #log.debug("getContractSpecsForContractType(): startTableLoc == " +
    #          str(startTableLoc))

    # Verify the tag was found.
    if (startTableLoc == -1):
        log.error("Couldn't find start tag of table for contract type " +
                  contractType)
        sys.exit(1)
    
    # Find startTableStr, starting from location 'startTableLoc'
    endTableLoc = lowercaseData.find(endTableStr, startTableLoc)
    
    #log.debug("getContractSpecsForContractType(): endTableLoc == " +
    #          str(endTableLoc))
    
    # Verify the tag was found.
    if (endTableLoc == -1):
        log.error("Couldn't find end tag of table for contract type " +
                  contractType)
        sys.exit(1)
    
    tableData = data[startTableLoc:endTableLoc]
    
    #log.debug("getContractSpecsForContractType(): Table data is: ", tableData)
    
    # Split by table rows.
    tableRows = tableData.split("<tr")
    
    # Chop off the first item if it doesn't have a closing </tr>.
    if len(tableRows) > 0:
        if (tableRows[0].find("</tr>") == -1):
            tableRows = tableRows[1:]
    
    #log.debug("getContractSpecsForContractType(): Split the table into " +
    #          str(len(tableRows)) + " rows (including heading)")
    
    rowList = []
        
    for row in tableRows:
        # Get rid of everything up to the first >, since it is still part
        # of the <tr tag.
        row = row[row.find(">")+1:]
        #log.debug("getContractSpecsForContractType(): Row is: " + row)
        dataList = extractTableDataFromRow(row)
        rowList.append(dataList)
    
    # Verify the first row list contains the headings we are expecting.
    if len(rowList) > 0:
        headingsData = rowList[0]

        # Make the row list only contain the non-header rows.
        rowList = rowList[1:]

        # Verify number of columns.
        if (len(headingsData) != 9):
            log.error("Number of columns in the table does not " +
                      "match what we are expecting.")
            sys.exit(1)
        elif (headingsData[0] != "Contract Name" and
              headingsData[1] != "Symbol" and
              headingsData[2] != "Months" and
              headingsData[3] != "Tick Size" and
              headingsData[4] != "Quoted Units" and
              headingsData[5] != "Trading Unit" and
              headingsData[6] != "Min Fluc" and
              headingsData[7] != "Init Margin" and
              headingsData[8] != "Maint Margin"):
            # Column names don't match what we expect them to be.
            log.error("Column names do not match what we expect.")
            sys.exit(1)
        else:
            log.debug("getContractSpecsForContractType(): Good, column " +
                      "names match what we expect.")
    
    for row in rowList:
        # Verify number of columns.
        if (len(row) != 9):
            log.error("Number of columns in the row entry does not " +
                      "match what we are expecting.")
            sys.exit(1)
        
        contractName = row[0]
        symbol = row[1]
        months = row[2]
        tickSize = row[3]
        quotedUnits = row[4]
        tradingUnit = row[5]
        minFluc = row[6]
        initMargin = row[7]
        maintMargin = row[8]
    
        months = str.strip(months)
        if months == "All 12":
            months = "FGHJKMNQUVXZ"
        
        if months != "":
            log.debug("getContractSpecsForContractType(): Creating new " +
                      "ContractSpecs for symbol " + symbol)
            cs = ContractSpecs(contractName,
                               symbol,
                               months,
                               tickSize,
                               quotedUnits,
                               tradingUnit,
                               minFluc,
                               initMargin,
                               maintMargin)
            contractSpecs.append(cs)
        else:
            log.warning("Found contract name '" + contractName + 
                        "' without months specified.  Skipping.")
    
    log.debug("Returning " + str(len(contractSpecs)) + " number of " + 
              "ContractSpecs")

    return contractSpecs

##############################################################################
# Subroutine: createAndSetupLogger(options)
#
# Description: Creates the logger and returns it to the caller.
#
# Input:
#    logLevel     - String with the log level.  Valid options are debug,
#                   info, warning, error, critical.
#
def createAndSetupLogger(logLevel):
    # Create a logger
    logger = logging.getLogger("ScraperLogger")

    # Set the log level.
    LEVELS = {'debug': logging.DEBUG,
              'info': logging.INFO,
              'warning': logging.WARNING,
              'error': logging.ERROR,
              'critical': logging.CRITICAL}
    level = LEVELS.get(logLevel, logging.INFO)
    logger.setLevel(level)

    # Create a 'logs' subdirectory if it doesn't already exist.
    logDir = os.path.join(sys.path[0], "logs")
    try:
        os.makedirs(logDir)
    except os.error as e:
        if e.errno != errno.EEXIST:
            raise

    # Set the log file.
    logFile = os.path.join(logDir, "scraper.log")

    # Create a handler, that rotates at 10 MB files with a backlog of 10 files.
    handler = logging.handlers.RotatingFileHandler(logFile,
                                                   maxBytes=10485760,
                                                   backupCount=10)
    
    # Create and set formatter.
    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
    handler.setFormatter(formatter)
    
    logger.addHandler(handler)
    
    return logger

##############################################################################
# Subroutine: processContracts(contractBasicInfos)
#
# Description:
#   Does the data harvesting/scraping for the list of ContractSpecs objects.
#
# Inputs:
#   contractBasicInfos     - List of ContractBasicInfo objects.
#
#   options.dataDirectory - Global variable string that holds the location of
#                           the data directory.
#
def processContracts(contractBasicInfos):
    global log
    global options

    log.debug("processContracts(contractBasicInfos)")

    for basicInfo in contractBasicInfos:
        # Contract name.
        name = basicInfo.name
        
        # Symbol in uppercase
        symbol = basicInfo.symbol.upper()

        log.debug("Setting up directory for contract with symbol " + symbol)
        
        # Contract directory to write files to.
        contractDir = setupDirectoryForContract(symbol)

        # Get information for each contract.
        contractInfos = getContractInfos(basicInfo)

        for contractInfo in contractInfos:
            log.info("Processing contract: " + contractInfo.toString())
            priceBars = getPriceBarsFromContractInfo(contractInfo)
            if len(priceBars) > 0: 
                startDate = priceBars[-1].getUSDateStr()
                endDate = priceBars[0].getUSDateStr()

                log.info("Obtained " + str(len(priceBars)) + 
                         " pricebars between dates " + startDate + 
                         " and " + endDate)
            else:
                log.info("Obtained 0 pricebars.")

            # Update the contract text file with price data.
            updateContractTextFile(contractInfo, priceBars)

            # Sleep for a bit so that we don't hammer the server, and
            # so it looks more like a human is accessing the site.
            sleepRequiredTime()

        log.info("Completed futures contracts for symbol " + symbol)

        # Sleep for a bit so that we don't hammer the server, and
        # so it looks more like a human is accessing the site.
        sleepRequiredTime()

        

##############################################################################
#
# Returns True on success, False on failure.
#
def updateContractTextFile(contractInfo, priceBars):
    global log
    global options
    
    successFlag = False
    
    # Only continue if there is at least 1 pricebar.
    if len(priceBars) < 1:
        log.warning("Tried to run updateContractTextFile() with " +
                    "an empty list of PriceBars!")
        successFlag = True
        return successFlag
    columnNamesStr = priceBars[0].columnNames()
    
    # This is the directory that we will be writing to.  It should be
    # created at this point in the script.
    contractDir = os.path.join(options.dataDirectory,
                               contractInfo.contractSymbol)
    log.debug("updateContractTextFile(): contractDir == " + contractDir)

    # Assemble what the filename should look like.  This format is the
    # format used by PriceData.
    #
    # <Symbol><2-digit-year><Month-Letter>.txt
    #
    filename = ""
    if (contractInfo.contractMonth != "Daily" and
        contractInfo.contractMonth != "Weekly" and
        contractInfo.contractMonth != "Monthly"):
        filename = contractInfo.contractSymbol + \
                   contractInfo.yearAs2DigitString() + \
                   contractInfo.contractMonth + ".txt"
    else:
        filename = contractInfo.contractSymbol + "_" + \
                   contractInfo.contractMonth + ".txt"
        
    log.debug("updateContractTextFile(): filename == " + filename)

    fullFilename = os.path.join(contractDir, filename)
    log.debug("updateContractTextFile(): fullFilename == " + fullFilename)

    # For iterating through the price bars.
    log.debug("priceBars has length " + str(len(priceBars)))
    
    filePriceBars = []
    
    # Initialize the file handle to the CSV data file.
    fh = None
    lineNumber = 0
    if os.path.exists(fullFilename):
        # File exists already.  Update it.
        log.info("File " + fullFilename +
                 " currently exists.  Doing file update ...")
        fh = open(fullFilename, "rt", encoding="utf8")
        line = fh.readline()
        lineNumber += 1
        line = line.strip("\r\n")
        log.debug("First line: " + line)
        if line != columnNamesStr:
            log.error("First line doesn't have matching column names " +
                      "as expected.  We see: " + line +
                      ", and we expected: " + columnNamesStr)
            successFlag = False
            return successFlag
        
        line = fh.readline()
        log.debug("Second line: " + line)
        lineNumber += 1
        while(line != ""):
            line = line.strip("\r\n")
            fields = line.split(",")
            if len(fields) != PriceBar.NumFieldsExpected:
                log.error("Encountered line " + str(lineNumber) +
                          " with the wrong number of fields.  Line: " +
                          line + ".")
                successFlag = False
                return successFlag
            dateStr = fields[0]
            openPrice = fields[1]
            highPrice = fields[2]
            lowPrice = fields[3]
            closePrice = fields[4]
            volume = fields[5]
            openInt = fields[6]
            
            splitDate = dateStr.split("/")
            month = splitDate[0]
            day = splitDate[1]
            year = splitDate[2]

            filePriceBar = PriceBar(year, month, day,
                                    openPrice, highPrice, lowPrice, closePrice,
                                    volume, openInt)
            filePriceBars.append(filePriceBar)

            # Read another line for the while loop.
            line = fh.readline()
            lineNumber += 1

        # We're done reading in all the File PriceBars.  Close the file handle.
        fh.close()

        log.debug("Done reading in the file.")
        log.debug("filePriceBars has length " + str(len(filePriceBars)))
        
        # We have all the file's pricebars in filePriceBars now.  If we find 
        for bar in priceBars:
            barDate = bar.date()

            replacedFlag = False
            for i in range(len(filePriceBars)):
                if filePriceBars[i].date() == barDate:
                    #log.debug("Found overlapping date " + str(barDate) +
                    #         " on line " + str(i+1) + ".  Replacing this bar.")
                    filePriceBars[i] = bar
                    replacedFlag = True

            if replacedFlag == False:
                # The date of 'bar' was not found in any of the
                # PriceBars in filePriceBars.  Append 'bar', so that
                # filePriceBars has it in its dataset.
                
                #log.debug("Bar " + bar.toString() + " wasn't found in the " +
                #          "file.  Appending it to the file PriceBar dataset.")
                filePriceBars.append(bar)
                
        # filePriceBars includes all bars from priceBars now, and also
        # has history that priceBars may not have.  We will save
        # filePriceBars as priceBars and work off of priceBars from
        # here.
        priceBars = filePriceBars

    else:
        # File does not exist.  Create it.
        log.info("File " + fullFilename + " does not exist.  " +
                 "Creating it and populating ...")

    # Go through and sort priceBars.
    sortedBars = sorted(priceBars, key=PriceBar.date)
    
    fh = open(fullFilename, "wt", encoding="utf8", newline='', closefd=True)
    endl = '\r\n'

    # Write as the first line, column names of the CSV file.
    fh.write(columnNamesStr + endl)
    for bar in sortedBars:
        fh.write(bar.toPriceBarCSVStr() + endl)
    fh.close()
    successFlag = True

    return successFlag
    
    
def sleepRequiredTime():
    global minimumSleepTimeSec
    global sleepTimeVarianceSec

    # Essentially checking to see if both are set to zero.
    if minimumSleepTimeSec < 0.1 and sleepTimeVarianceSec < 0.1:
        # Don't sleep at all.
        return
    
    # Get a random float value between 0.0 and 1.0
    randomNum = random.random()

    log.debug("Generated random number: " + str(randomNum))

    sleepTimeSec = minimumSleepTimeSec + (sleepTimeVarianceSec * randomNum)

    log.info("Sleeping for " + str(sleepTimeSec) +
              " seconds to simulate human web-browsing activity")
    time.sleep(sleepTimeSec)


##############################################################################
def getPriceBarsFromContractInfo(contractInfo):
    global log
    global defaultHttpHeaders

    log.info("Getting pricebar data for contract: " + contractInfo.toString())

    # Returned pricebars.
    priceBars = []

    webpage = contractInfo.getUrl()

    log.debug("getPriceBarsFromContractInfo(): Setting up HTTP connection " +
              "to " + webpage)

    opener = urllib.request.build_opener()
    request = urllib.request.Request(webpage)
    opener.addheaders = defaultHttpHeaders

    log.debug("getPriceBarsFromContractInfo(): Opening HTTP request.")

    openedRequestFlag = False
    while openedRequestFlag == False:
        try:
            response = opener.open(request)
            openedRequestFlag = True
        except URLError as err:
            log.error("Caught a URLError: " + err)
            openedRequestFlag = False
            log.info("Retrying HTTP request in 60 seconds.")
            time.sleep(60)


    log.debug("getPriceBarsFromContractInfo(): Reading HTTP response.")
    data = str(response.read())

    # Remove converted newline characters.
    data = data.replace('\\r\\n', '')
    data = data.replace('\\n', '')
    
    log.debug("getPriceBarsFromContractInfo(): data obtained is: " + \
              "***{}***".format(data))

    # Check for a bad contract month or data.
    errStr = "Note: The requested contract month is not trading and " + \
             "the related charts is therefore unavailable"
    if data.find(errStr) != -1:
        log.error("The webpage we requested (" + webpage + 
                  ") says that the contract month is not trading " +
                  "and therefore the data is unavailable.")
        # Return an empty list of pricebars.
        return priceBars

    # These are the starting and ending search strings that surround the data
    # we are seeking.
    startStr = "dataStr = new String(\\'"
    endStr = "\\')"

    startLoc = data.find(startStr)
    if (startLoc != -1):
        endLoc = data.find(endStr, startLoc)
        if (endLoc != -1):

            # This is the '|' delimited text that holds date, open, 
            # high, low, close, volume and open interest for all the 
            # pricebars available.
            priceDataStr = data[startLoc + len(startStr):endLoc]
            log.debug("Pure price data is: " + priceDataStr)
            priceDatas = priceDataStr.split("|")

            # Process each price bar text.
            for p in priceDatas:
                fields = p.split(",")

                # Verify correct number of fields.
                if len(fields) == PriceBar.NumFieldsExpected:
                    date = fields[0] 
                    open = fields[1]
                    high = fields[2]
                    low  = fields[3]
                    close = fields[4]
                    volume = fields[5]
                    openInterest = fields[6]

                    # Process the date fields into month, day and year.
                    dateFields = date.split("/")
                    if len(dateFields) != 3:
                        log.error("Date fields are incorrectly formatted " +
                                  "in string " + date)
                        break
                    month = dateFields[0]
                    day = dateFields[1]
                    year = dateFields[2]

                    # Create a PriceBar object for the data obtained.
                    priceBar = PriceBar(year, month, day,
                                        open, high, low, close, volume,
                                        openInterest)

                    #log.debug("Created price bar with data: " + 
                    #          priceBar.toString())

                    # Append to the list of PriceBar objects.
                    priceBars.append(priceBar)
                else:
                    log.error("Price data bar has incorrect number " +
                              "of fields: " + p + ".  Skipping price bar.")
        else:
            log.error("Could not find closing parenthesis for price data.")
    else:
        log.error("Could not find price data in the html page returned.")

    log.debug("Returning " + str(len(priceBars)) + " number of pricebars.")
    return priceBars

##############################################################################
# Subroutine: getContractInfos(contractBasicInfo)
#
# Description:  
#    Obtains a list of ContractInfo objects for the given basic contract name
#    and symbol by making a HTTP get.
#
# Arguments:
#    contractBasicInfo   - Object of type ContractBasicInfo.
#
def getContractInfos(contractBasicInfo):
    log.info("Getting list of futures contracts for symbol " +
             contractBasicInfo.symbol + ".")

    # Returned ContractInfos list.
    contractInfos = []

    symbol = contractBasicInfo.symbol
    contractNameStr = contractBasicInfo.name

    webpage = "http://tfc-charts.w2d.com/chart/" + symbol

    log.debug("getContractInfos(): Setting up HTTP connection " +
              "to " + webpage)

    opener = urllib.request.build_opener()
    request = urllib.request.Request(webpage)
    opener.addheaders = defaultHttpHeaders

    log.debug("getContractInfos(): Opening HTTP request.")
    response = opener.open(request)

    log.debug("getContractInfos(): Reading HTTP response.")
    data = str(response.read())

    # Remove converted newline characters.
    data = data.replace('\\r\\n', '')
    data = data.replace('\\n', '')
    
    log.debug("getContractInfos(): data obtained is: " + data)
    
    # Verify we were not redirected to the menu page.
    searchStr = "Commodity Futures Charts & Futures Quotes Menu"
    if data.find(searchStr) != -1:
        log.error("An invalid contract symbol must have been queried " +
                  "(symbol == " + symbol + ").  " +
                  "We were redirected to the main contracts menu.  " +
                  "Skipping this contract symbol.")
        # Return an empty list of ContractInfos.
        return contractInfos

    # This is the starting point of getting the info we want.
    #searchStr = "Additional Months:"
    searchStr = "<select Name=\"fSelect\""
    searchStrLoc = data.find(searchStr)

    if searchStrLoc == -1:
        # There is no drop-down for the contract month.
        # Confirm there is no text for a heading for the contract
        # month.
        if data.find("Contract month:") == -1:
            # Assume that this is a stock or commodity index that has
            # no contract months.
            log.debug("Assuming that this is a stock or commodity index," + \
                      " and thus it has no contract months.")
            
            # Create a ContractInfo for this, for daily, weekly and
            # monthly data.
            
            # URL path will be in the format: "/chart/XX/X", where XX
            # is the symbol, and X stands for daily bars.  If X is a
            # 'W' then it is weekly, and if it is an 'M' then it is
            # monthly.
            descriptionStr = "Daily"
            urlPathStr = "/chart/{}/X".format(symbol)
            contractInfo = ContractInfo(contractNameStr, symbol, 
                                        descriptionStr, urlPathStr)
            contractInfos.append(contractInfo)
            
            descriptionStr = "Weekly"
            urlPathStr = "/chart/{}/W".format(symbol)
            contractInfo = ContractInfo(contractNameStr, symbol, 
                                        descriptionStr, urlPathStr)
            contractInfos.append(contractInfo)

            descriptionStr = "Monthly"
            urlPathStr = "/chart/{}/M".format(symbol)
            contractInfo = ContractInfo(contractNameStr, symbol, 
                                        descriptionStr, urlPathStr)
            contractInfos.append(contractInfo)

            # Return the list of ContractInfos with only this inside
            # since there are no other 'contracts'.
            return contractInfos
        else:
            log.error("Unable to find list of available contracts " + \
                      "from the HTML.  data was: ***{}***".format(data))
            
            # Return an empty list of ContractInfos.
            return contractInfos

    # Continue to find and extract the data required.
    #searchStr = "<select"
    #startSelectLoc = data.find(searchStr, searchStrLoc)
    startSelectLoc = searchStrLoc
    searchStr = "</select"
    endSelectLoc = data.find(searchStr, startSelectLoc)

    cutString = data[startSelectLoc:endSelectLoc]

    optionsValues = cutString.split("<option ")

    # If there's at least one option (option split by the opening tag), 
    # then drop the first index value, because it contains the '<select'
    # opening tag.
    if len(optionsValues) > 1:
        optionsValues = optionsValues[1:]

    for contractText in optionsValues:
        log.debug("getContractInfos(): Contract text is ***" +
                  contractText + "***")
        
        match = re.search(r""".*?value="([^"]*?)">([^<]*?)</option>""",
                          contractText)
        
        # Comment below with double-quote to get emacs to parse for
        # colors correctly.
        #"

        if (match):
            # Here we are looking for the format "/chart/XX/MM", where XX is
            # the symbol, and MM is the month of contract.
            
            urlPathStr = "/chart" + match.groups()[0]
            monthYearStr = match.groups()[1].strip()

            # If the url is in the format "/chart/XX/", then we should
            # ignore it, since it doesn't have the format we want.
            if urlPathStr.endswith("/") and urlPathStr.count("/") == 3:
                # Ignore.
                log.debug("getContractInfos(): Ignoring urlPathStr == " + 
                          urlPathStr + ", monthYearStr == " + monthYearStr)
            else:
                log.debug("getContractInfos(): Found urlPathStr == " + 
                          urlPathStr + ", monthYearStr == " + monthYearStr)
                contractInfo = ContractInfo(contractNameStr, symbol, 
                                            monthYearStr, urlPathStr)
                contractInfos.append(contractInfo)

        else:
            log.error("getContractInfos(): Failed to parse a contract " +
                      "month and url from this select option: " + 
                      contractText)

    # Add contracts for weekly and monthly charts.

    # URL path will be in the format: "/chart/XX/X", where XX is the
    # symbol, and X stands for the time period.  If X is a 'W' then it
    # is weekly, and if it is an 'M' then it is monthly.
    descriptionStr = "Weekly"
    urlPathStr = "/chart/{}/W".format(symbol)
    contractInfo = ContractInfo(contractNameStr, symbol, 
                                descriptionStr, urlPathStr)
    contractInfos.append(contractInfo)

    descriptionStr = "Monthly"
    urlPathStr = "/chart/{}/M".format(symbol)
    contractInfo = ContractInfo(contractNameStr, symbol, 
                                descriptionStr, urlPathStr)
    contractInfos.append(contractInfo)

    
    log.debug("getContractInfos(): Returning " + str(len(contractInfos)) + 
              " number of contractInfos")
    return contractInfos

##############################################################################

def setupDirectoryForContract(symbol):
    global log
    global options
    dataDirectory = options.dataDirectory

    log.debug("Data directory is: " + dataDirectory)

    # Check to see if the directory for the symbol has been
    # created already.

    # This is the directory we will write to.
    contractDir = os.path.join(dataDirectory, symbol)

    log.debug("Contract directory is: " + contractDir)
    
    # Check to see if the directory path exists.
    if not os.path.exists(contractDir):
        # Create the directory since it doesn't exist.
        log.info("Directory for " + symbol +
                 " does not exist.  Creating it...")
        os.makedirs(contractDir)
        
    # Verify that the directory 'contractDir' is actually a directory.
    if not os.path.isdir(contractDir):
        log.error("Cannot use '" + contractDir + "' as the data " +
                  "directory because it is not a directory.")
        sys.exit(1)
    else:
        log.debug("Contract directory " + contractDir + " exists.")
        
    return contractDir

##############################################################################
# Subroutine: writeDataDirectoryListingFile()
#
# Description: Writes file directoryListing.txt with descriptions of
# directories in the data directory.
#
def writeDataDirectoryListingFile():
    global log
    global options

    dataDirectory = options.dataDirectory
    directoryListingFilename = "directoryListing.txt"

    fullFilename = os.path.join(dataDirectory, directoryListingFilename)

    log.debug("writeDataDirectoryListingFile(): fullFilename == " +
              fullFilename)

    if not os.path.exists(fullFilename):
        fh = open(fullFilename, "at+", encoding="utf8",
                  newline='\r\n', closefd=True)
        fileContents = \
"""Symbol  Contract Description
------  --------------------------------------------------------
A6      Australian Dollar (CME)
AA      Amex 10 Most Active
AC      Ethanol (CBOT)
AD      Australian Dollar (CME)
AL      Aluminum (delisted by exchange) (COMEX)
AV      Amex 10 Most Active Volume
B6      British Pound (CME)
BA      Alberta Barley (ICE [WCE])
BC      Brent Crude Oil (IPE)
BP      British Pound (CME)
BU      Butter (CME)
BX      Cdn Bankers Accept. 3 Month (MFE)
CA      Canola (ICE [WCE])
CC      Cocoa (ICE [NYBOT])
CD      Canadian Dollar (CME)
CF      Coffee (ICE [NYBOT])
CG      10 Yr Govt of Canada Bonds (MFE)
CI      Continuous CRB Index (ICE [NYBOT])
CL_     Light Crude Oil (Pit) (NYMEX)
CN      Corn (CBOT)
CO      Light Crude Oil (Pit) (NYMEX)
CP      Copper High Grade (COMEX)
CR      CRB Index Futures (ICE [NYBOT])
CT      Cotton #2 (ICE [NYBOT])
CW      Wheat (CBOT)
D6      Canadian Dollar (CME)
DA      BFP Milk (CME)
DG      Gold (COMEX)
DJ      Dow Jones Industrial Average (CBOT)
DL      BFP Milk (CME)
DW      Dow Jones Industrial Average
E6      Euro (CME)
E7      Euro Emini (CME)
EC      Euro (CME)
ED      Euro Dollar (CME)
EM      Libor 1 Month (CME)
ES      S&P 500 E-mini (CME)
EY      Euro Yen (CME)
FC      Feeder Cattle (CME)
FF      Fed Funds 30 Day (CBOT)
FV      Treasury Notes 5 Year (CBOT)
GD      Gold (COMEX)
GE      Euro Dollar (CME)
GF      Feeder Cattle (CME)
GH      Libor 1 Month (CME)
GL      Gilts Long (LIFFE)
GO      Gas - Oil (IPE)
GX      Goldman Sachs Commodity Index (CME)
GX_     Goldman Sachs Commodity Index (CME)
HE      Lean Hogs (CME)
HG_     Copper High Grade (COMEX)
HO      Heating Oil (NYMEX)
HO_     Heating Oil (NYMEX)
J6      Japanese Yen (CME)
J7      Japanese Yen Emini (CME)
JM      PJM Electricity (NYMEX)
JY      Japanese Yen (CME)
KE      Wheat (Kansas) (KCBT)
KK      New Zealand Dollar (CME)
KW      Wheat (Kansas) (KCBT)
LC      Live Cattle (CME)
LE      Live Cattle (CME)
LH      Lean Hogs (CME)
LS      Lumber (CME)
LU      Lumber (CME)
LZ      London FT-SE 100 (LIFFE)
M6      Mexican Peso (CME)
MD      S&P Midcap 400 (CME)
ME      Emini S&P Midcap 400 (CME)
MP      Mexican Peso (CME)
MW      Hard Red Spring Wheat (Electronic) (MGE)
N6      New Zealand Dollar (CME)
NA      NASDAQ Composite Index
NC      NYSE Composite Futures (ICE [NYBOT])
ND      Nasdaq 100 (CME)
ND_     Nasdaq 100 (CME)
NG      Natural Gas (NYMEX)
NG_     Natural Gas (NYMEX)
NK      Nikkei 225 (CME)
NO      Treasury Notes 10 Year (CBOT)
NQ      Nasdaq 100 E-mini (CME)
NW      10 Year Interest Rate Swap (CBOT)
NY      Nikkei 225 (CME)
OA      Oats (CBOT)
OJ      Orange Juice (ICE [NYBOT])
PA      Palladium (NYMEX)
PA_     Palladium (NYMEX)
PB      Frozen Pork Bellies (CME)
PD      Frozen Pork Bellies (CME)
PL      Platinum (NYMEX)
PL_     Platinum (NYMEX)
PR      Propane (delisted by exchange Sept 18th) (NYMEX)
QC      Copper miNY (Globex) (COMEX)
QG      Natural Gas miNY (NYMEX)
QI      Silver miNY (Globex) (COMEX)
QM      Crude Oil EmiNY (NYMEX)
QO      Gold miNY (Globex) (COMEX)
R1      Russell 1000 Mini (ICE [NYBOT])
RB      NYH RBOB Unleaded Gas (NYMEX)
RB_     NYH RBOB Unleaded Gas (NYMEX)
RC      CRB Index 
RI      Rough Rice (CBOT)
RJ      Russell 2000 Mini (ICE [NYBOT])
RW      Russell 2000 Index
S6      Swiss Franc (CME)
SB      Soybeans (CBOT)
SD      Sugar #16 (ICE [NYBOT])
SE      Sugar #14 (delisted by exchange) (ICE [NYBOT])
SF      Swiss Franc (CME)
SM      Soybean Meal (CBOT)
SO      Soybean Oil (CBOT)
SP      S&P 500 (CME)
SP_     S&P 500 (CME)
SR      10 Year Interest Rate Swap (CBOT)
SS      Short Sterling (LIFFE)
SU      Sugar #11 (ICE [NYBOT])
SV      Silver 5000 oz (COMEX)
SV_     Silver 5000 oz (COMEX)
SW      S&P 500 Index 
SX      S&P Canada 60 Index (MFE)
TN      Treasury Notes 2 Year (CBOT)
TO      Russell 2000 (ICE [NYBOT])
TR      Treasury Bonds 30 Year (CBOT)
TS      Toronto S&P 60
US      US Dollar Index (ICE [NYBOT])
UX      Uranium (Globex) (NYMEX)
XC      Corn Mini (CBOT)
XK      Soybeans Mini (CBOT)
XN      Corn Mini (CBOT)
XS      Soybeans Mini (CBOT)
XW      Mini Wheat (CBOT)
XW_     Mini Wheat (CBOT)
YG      Gold, NY Mini (CBOT)
YI      Silver NY Mini (CBOT)
YM      Dow Jones E-mini ($5) (Globex) (CBOT)
ZB      Treasury Bonds 30 Year (CBOT)
ZC      Corn (CBOT)
ZD      Dow Jones Industrial Average (CBOT)
ZF      Treasury Notes 5 Year (CBOT)
ZG      Gold 100 oz. (CBOT)
ZI      Silver 5000 oz (CBOT)
ZK      Ethanol (CBOT)
ZL      Soybean Oil (CBOT)
ZM      Soybean Meal (CBOT)
ZN      Treasury Notes 10 Year (CBOT)
ZO      Oats (CBOT)
ZQ      Fed Funds 30 Day (CBOT)
ZR      Rough Rice (CBOT)
ZS      Soybeans (CBOT)
ZT      Treasury Notes 2 Year (CBOT)
ZW      Wheat (CBOT)
"""
        fh.write(fileContents)
        fh.close()
                              

##############################################################################
# Subroutine: main()
#
# Description:
#     Does the setup and scraping of data.
#
def main():
    # Record a timestamp for when we started.
    startTimestamp = datetime.datetime.today()

    # Parse command-line arguments.
    (options, args) = parseCommandlineArgs()
    
    # Set up the logger.
    global log
    log = createAndSetupLogger(options.logLevel)

    log.info("##########################################");
    log.info("# Starting " + sys.argv[0]);
    log.info("##########################################");

    log.info("Log level is set to " + options.logLevel)
    
    global minimumSleepTimeSec
    minimumSleepTimeSec = options.minSleepSec
    log.info("Initializing minimum sleep time between HTTP calls to " +
             str(minimumSleepTimeSec) + " seconds.")
    
    global sleepTimeVarianceSec
    sleepTimeVarianceSec = options.sleepVarianceSec
    log.info("Initializing sleep time variance between HTTP calls to " +
             str(sleepTimeVarianceSec) + " seconds.")
    

    ## Get the HTML for all contracts.
    #html = getSupportedContractsHTML()
    #
    ## Get ContractSpecs objects for each contract type.
    #contractType = "Agricultural Contract Specifications"
    #agriculturalContracts = getContractSpecsForContractType(html,
    #                                                        contractType)
    #contractType = "Currencies Contract Specifications"
    #currenciesContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Livestock Contract Specifications"
    #livestockContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Food/Fiber/Softs Contract Specifications"
    #foodContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Financials Contract Specifications"
    #financialContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Index Contract Specifications"
    #indexContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Oil/Energy Contract Specifications"
    #energyContracts = getContractSpecsForContractType(html, contractType)
    #contractType = "Metals Contract Specifications"
    #metalsContracts = getContractSpecsForContractType(html, contractType)


    
    # Get the HTML for all supported contract chart symbols.
    #html = getSupportedContractSymbolsHTML()
    html = getHtmlFromLocalFile()

    # Get ContractSpecs objects for each contract type.
    contractType = "GRAINS, CEREAL / OILSEED"
    agriculturalContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)

    #contractType = "CATTLE / HOGS / MEAT PRODUCTS"
    contractType = "CATTLE, HOGS, MEAT PRODUCTS"
    livestockContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)

    contractType = "PETROLEUM / ENERGY FUTURES"
    energyContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)

    contractType = "SOFTS, FOOD AND FIBER"
    foodContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)

    contractType = "METALS"
    metalsContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)

    contractType = "CURRENCIES"
    currenciesContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)
    
    contractType = "INDEX FUTURES"
    indexContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)
    
    contractType = "INTEREST / FINANCIAL FUTURES"
    financialContractBasicInfos = \
        getContractBasicInfosForType(html, contractType)
    
    contractType = "LESS POPULAR INTEREST / FINANCIAL FUTURES"
    for bi in getContractBasicInfosForType(html, contractType):
        financialContractBasicInfos.append(bi)
    
    #contractType = "COMMODITY &amp; STOCK INDEXES"
    contractType = "COMMODITY & STOCK INDEXES"
    indexesBasicInfos = getContractBasicInfosForType(html, contractType)

    # Do a little cleanup on the Indexes set of BasicInfo's, since the
    # site seems to want the 'Daily' hyperlink to link directly to the
    # chart webpage, instead of to a intermediary page like all the
    # futures contracts.
    i = 0
    for i in range(len(indexesBasicInfos)):
        #log.debug("Symbol is: " + indexesBasicInfos[i].symbol)
        slashLoc = indexesBasicInfos[i].symbol.find("/")
        #log.debug("slashLoc == " + str(slashLoc))
        if slashLoc != -1 and slashLoc != 0:
            indexesBasicInfos[i].symbol = \
                                        indexesBasicInfos[i].symbol[0:slashLoc]

    if (options.all == True):
        log.info("Processing agricultural contracts ...")
        processContracts(agriculturalContractBasicInfos)
        log.info("Done processing agricultural contracts.")

        log.info("Processing currency contracts ...")
        processContracts(currenciesContractBasicInfos)
        log.info("Done processing currency contracts.")

        log.info("Processing livestock contracts ...")
        processContracts(livestockContractBasicInfos)
        log.info("Done processing livestock contracts.")

        log.info("Processing food/fiber/softs contracts ...")
        processContracts(foodContractBasicInfos)
        log.info("Done processing food/fiber/softs contracts.")

        log.info("Processing financial contracts ...")
        processContracts(financialContractBasicInfos)
        log.info("Done processing financial contracts.")

        log.info("Processing index contracts ...")
        processContracts(indexContractBasicInfos)
        log.info("Done processing index contracts.")

        log.info("Processing energy contracts ...")
        processContracts(energyContractBasicInfos)
        log.info("Done processing energy contracts.")
        
        log.info("Processing metals contracts ...")
        processContracts(metalsContractBasicInfos)
        log.info("Done processing metals contracts.")

        log.info("Processing commodity and stock indexes ...")
        processContracts(indexesBasicInfos)
        log.info("Done processing commodity and stock indexes.")

    elif (options.agricultural == True):
        log.info("Processing agricultural contracts ...")
        processContracts(agriculturalContractBasicInfos)
        log.info("Done processing agricultural contracts.")
    elif (options.currencies == True):
        log.info("Processing currency contracts ...")
        processContracts(currenciesContractBasicInfos)
        log.info("Done processing currency contracts.")
    elif (options.livestock == True):
        log.info("Processing livestock contracts ...")
        processContracts(livestockContractBasicInfos)
        log.info("Done processing livestock contracts.")
    elif (options.food == True):
        log.info("Processing food/fiber/softs contracts ...")
        processContracts(foodContractBasicInfos)
        log.info("Done processing food/fiber/softs contracts.")
    elif (options.financial == True):
        log.info("Processing financial contracts ...")
        processContracts(financialContractBasicInfos)
        log.info("Done processing financial contracts.")
    elif (options.index == True):
        log.info("Processing index contracts ...")
        processContracts(indexContractBasicInfos)
        log.info("Done processing index contracts.")
    elif (options.energy == True):
        log.info("Processing energy contracts ...")
        processContracts(energyContractBasicInfos)
        log.info("Done processing energy contracts.")
    elif (options.metals == True):
        log.info("Processing metals contracts ...")
        processContracts(metalsContractBasicInfos)
        log.info("Done processing metals contracts.")
    elif (options.indexes == True):
        log.info("Processing commodity and stock indexes ...")
        processContracts(indexesBasicInfos)
        log.info("Done processing commodity and stock indexes.")
    else:
        log.error("No valid options were specified for what type(s) of " +
                  "contracts data to scrape.")

    # Write a text file in the data directory that will help the user
    # translate what the symbols are to the full contract description.
    writeDataDirectoryListingFile()
    
    # Record a timestamp for when we started.
    finishTimestamp = datetime.datetime.today()

    log.info("Execution completed in " + str(finishTimestamp - startTimestamp))
    
    # Report that we are exiting.
    exitStr = "Done.  Now exiting."
    log.info(exitStr)
    print(exitStr)


##############################################################################
def getHtmlFromLocalFile():
    """Opens a file that holds the html to be parsed.
    This is for debugging/testing purposes only.
    """

    log.debug("Entered getHtmlFromLocalFile().")

    # Run wget to get the html file.
    #
    # The server sometimes returns different versions of menu.html,
    # which makes our parsing much harder.  Here we just assume that
    # menu.html (of the right version), is in the logs directory.
    # 
    #log.debug("Changing to directory 'logs'")
    #os.chdir("logs")
    #log.debug("Calling 'wget http://tfc-charts.w2d.com/menu.html' ...")
    #call(["wget", "http://tfc-charts.w2d.com/menu.html"])
    #log.debug("Changing to directory '../'")
    #os.chdir("../")

    filename = "logs/menu.html"

    log.debug("Reading file: " + filename)
    
    data = ""
    with open(filename, "r") as f:
        data = f.read()

    log.debug("Data read from file is: " + \
              "***{}***".format(str(data)))
    
    # Remove converted newline characters.
    data = str(data)
    data = data.replace('\\r\\n', '')
    data = data.replace('\\n', '')
    
    log.debug("Exiting getHtmlFromLocalFile().")
    
    return data
    
    
##############################################################################

if __name__ == "__main__":
    main()

##############################################################################
